#! /usr/bin/python2.5
# This file is part of SyncStar, free software released under GPL v2.
# Google AppEngine Python 2.5 source.
#
# wget -S -O- --post-data='[5,6,7]' http://127.0.0.1:8080/sync\?mydebug=1
# wget -S -O- --post-file=smsshort.txt http://127.0.0.1:8080/sync
#
# curl -d@webapp/errsmsphoneempty.txt -D- http://127.0.0.1:8080/sync
# (wget doesn't show the body of the response)
#
# Datastore data format changed to compressed msgsdata at r630.
#
# TODO: Make `appcfg.py update' deploy only a few .py and .yaml files.
#
# cp ahello1/out/production/ahello1/ahello1.apk webapp/static/syncstar.apk

import calendar
import cgi
import Cookie
import logging
import os
import random
import re
import time
import zlib

from google.appengine.api import memcache
from google.appengine.api import users
from google.appengine.ext import db
from google.appengine.ext import webapp
from google.appengine.ext.webapp.util import run_wsgi_app

# Production instance uses the built-in simplejson module instead.
import datastore_count
import db_cache
import msg_data

# Import from webapp/aalib, which is 2.2.1.
import aalib.simplejson as json

# AppEngine Python 2.5 has simplejson 2.1.3 without the fast C implementation
# (simplejson._speedups) built in. Only AppEngine Python 2.7 has:
# cPickle, C-accelerated json.
#import simplejson as json

NONWHITESPACE_RE = re.compile(r'\S')

NO_YES = ('no', 'yes')

TYPE_TO_STRING = {
    1: 'received',
    2: 'sent',
    3: 'draft',
    4: 'outbox',
    5: 'failed',
    6: 'queued',
}

is_development = os.environ.get('SERVER_SOFTWARE', '').startswith(
    'Development/')
if not is_development:
  datastore_count.register_db_count_hook()

WORDS = ['the', 'of', 'to', 'and', 'a', 'in', 'is', 'it', 'you', 'that', 'he',
         'was', 'for', 'on', 'are', 'with', 'as', 'I', 'his', 'they', 'be',
         'at', 'one', 'have', 'this', 'from', 'or', 'had', 'by', 'hot', 'but',
         'some', 'what', 'there', 'we', 'can', 'out', 'other', 'were', 'all',
         'your', 'when', 'up', 'use', 'word', 'how', 'said', 'an', 'each',
         'she', 'which', 'do', 'their', 'time', 'if', 'will', 'way', 'about',
         'many', 'then', 'them', 'would', 'write', 'like', 'so', 'these', 'her',
         'long', 'make', 'thing', 'see', 'him', 'two', 'has', 'look', 'more',
         'day', 'could', 'go', 'come', 'did', 'my', 'sound', 'no', 'most',
         'number', 'who', 'over', 'know', 'water', 'than', 'call', 'first',
         'people', 'may', 'down', 'side', 'been', 'now', 'find', 'any', 'new',
         'work', 'part', 'take', 'get', 'place', 'made', 'live', 'where',
         'after', 'back', 'little', 'only', 'round', 'man', 'year', 'came',
         'show', 'every', 'good', 'me', 'give', 'our', 'under', 'name', 'very',
         'through', 'just', 'form', 'much', 'great', 'think', 'say', 'help',
         'low', 'line', 'before', 'turn', 'cause', 'same', 'mean', 'differ',
         'move', 'right', 'boy', 'old', 'too', 'does', 'tell', 'sentence',
         'set', 'three', 'want', 'air', 'well', 'also', 'play', 'small', 'end',
         'put', 'home', 'read', 'hand', 'port', 'large', 'spell', 'add', 'even',
         'land', 'here', 'must', 'big', 'high', 'such', 'follow', 'act', 'why',
         'ask', 'men', 'change', 'went', 'light', 'kind', 'off', 'need',
         'house', 'picture', 'try', 'us', 'again', 'animal', 'point', 'mother',
         'world', 'near', 'build', 'self', 'earth', 'father', 'head', 'stand',
         'own', 'page', 'should', 'country', 'found', 'answer', 'school',
         'grow', 'study', 'still', 'learn', 'plant', 'cover', 'food', 'sun',
         'four', 'thought', 'let', 'keep', 'eye', 'never', 'last', 'door',
         'between', 'city', 'tree', 'cross', 'since', 'hard', 'start', 'might',
         'story', 'saw', 'far', 'sea', 'draw', 'left', 'late', 'run', "don't",
         'while', 'press', 'close', 'night', 'real', 'life', 'few', 'stop',
         'open', 'seem', 'together', 'next', 'white', 'children', 'begin',
         'got', 'walk', 'example', 'ease', 'paper', 'often', 'always', 'music',
         'those', 'both', 'mark', 'book', 'letter', 'until', 'mile', 'river',
         'car', 'feet', 'care', 'second', 'group', 'carry', 'took', 'rain',
         'eat', 'room', 'friend', 'began', 'idea', 'fish', 'mountain', 'north',
         'once', 'base', 'hear', 'horse', 'cut', 'sure', 'watch', 'color',
         'face', 'wood', 'main', 'enough', 'plain', 'girl', 'usual', 'young',
         'ready', 'above', 'ever', 'red', 'list', 'though', 'feel', 'talk',
         'bird', 'soon', 'body', 'dog', 'family', 'direct', 'pose', 'leave',
         'song', 'measure', 'state', 'product', 'black', 'short', 'numeral',
         'class', 'wind', 'question', 'happen', 'complete', 'ship', 'area',
         'half', 'rock', 'order', 'fire', 'south', 'problem', 'piece', 'told',
         'knew', 'pass', 'farm', 'top', 'whole', 'king', 'size', 'heard',
         'best', 'hour', 'better', 'true', 'during', 'hundred', 'am',
         'remember', 'step', 'early', 'hold', 'west', 'ground', 'interest',
         'reach', 'fast', 'five', 'sing', 'listen', 'six', 'table', 'travel',
         'less', 'morning', 'ten', 'simple', 'several', 'vowel', 'toward',
         'war', 'lay', 'against', 'pattern', 'slow', 'center', 'love', 'person',
         'money', 'serve', 'appear', 'road', 'map', 'science', 'rule', 'govern',
         'pull', 'cold', 'notice', 'voice', 'fall', 'power', 'town', 'fine',
         'certain', 'fly', 'unit', 'lead', 'cry', 'dark', 'machine', 'note',
         'wait', 'plan', 'figure', 'star', 'box', 'noun', 'field', 'rest',
         'correct', 'able', 'pound', 'done', 'beauty', 'drive', 'stood',
         'contain', 'front', 'teach', 'week', 'final', 'gave', 'green', 'oh',
         'quick', 'develop', 'sleep', 'warm', 'free', 'minute', 'strong',
         'special', 'mind', 'behind', 'clear', 'tail', 'produce', 'fact',
         'street', 'inch', 'lot', 'nothing', 'course', 'stay', 'wheel', 'full',
         'force', 'blue', 'object', 'decide', 'surface', 'deep', 'moon',
         'island', 'foot', 'yet', 'busy', 'test', 'record', 'boat', 'common',
         'gold', 'possible', 'plane', 'age', 'dry', 'wonder', 'laugh',
         'thousand', 'ago', 'ran', 'check', 'game', 'shape', 'yes', 'hot',
         'miss', 'brought', 'heat', 'snow', 'bed', 'bring', 'sit', 'perhaps',
         'fill', 'east', 'weight', 'language', 'among']


class UserData(db.Model):
  user = db.UserProperty(required=True)
  deleted = db.BooleanProperty(default=False, indexed=False)
  time_created = db.IntegerProperty(indexed=False)  # UNIX timestamp * 1000 (milliseconds).
  time_deleted = db.IntegerProperty(indexed=False)  # UNIX timestamp * 1000 (milliseconds).
  livemsgcount = db.IntegerProperty(default=0, indexed=False)
  deletedmsgcount = db.IntegerProperty(default=0, indexed=False)
  compressedsize = db.IntegerProperty(default=0, indexed=False)


class Message(db.Model):
  # The user_id is stored in the key_name of the parent UserData, so we don't
  # need the following field:
  #
  #   user = db.UserProperty(required=True, indexed=True)
  #
  # Oddly enough, this removal sped up webapp_test/sms_test.py by a factor of 6.

  # List of messages, serialized and compressed by msg_data.serialize_messages.
  # Since the maximum Datastore entity size is 1MB (let's assume that there is
  # 1K overhead), and 1 message is about 57 bytes compressed.
  # required=True makes it fail with an empty string.
  msgsdata = db.BlobProperty(default='', indexed=False)


class User(db.Model):
  """Helper model for converting an e-mail address to a user ID."""
  user = db.UserProperty(required=True)


def email_to_user(email):
  """Converts an e-mail address to a user ID.

  Implementation is based on the workaround
  http://stackoverflow.com/questions/816372/
  how-can-i-determine-a-user-id-based-on-an-email-address-in-app-engine

  Returns:
    A User object with a stable user_id string based on an email address, or
    None if the address is not a valid/existing Google account.
  """
  assert isinstance(email, basestring), type(email)
  key = User(user=users.User(email)).put()
  obj = User.get(key)
  if obj:
    obj.delete()
  if obj.user and obj.user.user_id():
    return obj.user
  else:
    return None


def format_mts(mts):
  """Format a millisecond timestamp as a Gregorian date and time.

  Args:
    mts: Millisecond timestamp: integer containing the number of milliseconds
      since the epoch.
  Returns:
    A string in GMT describing the date and time.
  """
  if mts is None or mts == '':
    return ''
  mts = int(mts)
  ms = mts % 1000
  if ms:
    ms = '.%03d' % ms
  else:
    ms = ''
  t = time.gmtime(mts / 1000)
  return '%04d-%02d-%02d %02d:%02d:%02d%s GMT' % (
      t.tm_year, t.tm_mon, t.tm_mday,
      t.tm_hour, t.tm_min, t.tm_sec, ms)


def get_updated_qstr(qstr, key, default_value, value):
  """Return query_string updated with key=value.

  Args:
    qstr: A URL query_string, may or may not start with '?'.
    key: The key to update.
    default_value: The default value for the key.
    value: The new value for the key.
  Returns:
    The updated URL query string: empty or starts with '?'.
  """
  if not isinstance(qstr, str):
    raise TypeError
  if not isinstance(key, str):
    raise TypeError
  assert key
  assert '=' not in key
  assert '&' not in key
  if default_value is not None:
    if not isinstance(default_value, str):
      raise TypeError
  if isinstance(value, bool):
    value = str(int(value))  # '0' or '1'.
  elif isinstance(value, (int, long)):
    value = str(value)
  elif not isinstance(value, str) and value is not None:
    raise TypeError

  # TODO: Do it with less string copies.
  qstr = qstr.strip('?&')
  if qstr:
    qstr = '&' + qstr
  i = 0
  pattern = '&' + key + '='
  output = []
  while True:
    j = qstr.find(pattern, i)
    if j < 0:
      break
    k = qstr.find('&', j + len(pattern))
    if k < 0:
      k = len(qstr)
    output.append(qstr[i : j])
    i = k
  output.append(qstr[i:])
  if value != default_value and value is not None:
    # TODO: Modify in place if there was a match.
    # TODO: Insert in alphabetical order to reduce URL ambiguity.
    output.append(pattern)
    output.append(value)
  qstr = ''.join(output)
  if len(qstr) > 1:
    return '?' + qstr[1:]
  else:
    return ''


def run_counted(count_name, function, *args, **kwargs):
  call = [(function, args, kwargs)]  # Allow early arg deletion within function.
  del function, args, kwargs
  if datastore_count.global_db_cost is None:
    return apply(*call.pop())
  start_cost = datastore_count.global_db_cost.copy()
  start_ts = time.time()
  is_success = False
  try:
    retval = apply(*call.pop())
    is_success = True
  finally:
    duration_ms = int((time.time() - start_ts) * 1000 + 0.5)
    this_cost = datastore_count.global_db_cost - start_cost
    logging.info('count name:%s time:%sms %s' % (
        count_name, duration_ms, this_cost.format_short()))
  return retval


def run_in_counted_transaction(count_name, function, *args, **kwargs):
  return run_counted(count_name, db.run_in_transaction, function,
                     *args, **kwargs)


def get_user_key(user):
  assert isinstance(user, users.User), type(user)
  assert user.user_id()
  return db.Key.from_path('UserData', user.user_id())


def update_user_data_entity(user, userinfo):
  assert isinstance(userinfo, dict)
  assert isinstance(user, users.User), type(user)
  assert user.user_id()
  assert user.email()
  key_name = user.user_id()
  parent_user_data = UserData.get_by_key_name(key_name)
  if parent_user_data is None:
    parent_user_data = UserData(key_name=key_name,
                                user=user,
                                time_created=int(time.time() * 1000 + .5),
                                **userinfo)
  else:
    parent_user_data.deleted = False
    for name in sorted(userinfo):
      setattr(parent_user_data, name, userinfo[name])
  parent_user_data.put()
  return parent_user_data.key()
  # TODO: Cache UserData in memcache, so /count can be served from the cache.


def compute_userinfo(messages, msgsdata):
  assert isinstance(msgsdata, str)
  msgcount = len(messages)
  deletedmsgcount = sum(1 for message in messages if message.get('deleted'))
  return {
      'livemsgcount': msgcount - deletedmsgcount,
      'deletedmsgcount': deletedmsgcount,
      'compressedsize':  len(msgsdata),
      'deleted': False,
  }


class MessageEntryEnv(db_cache.EntryEnv):
  __slots__ = ['user', 'user_id', 'mc_client', 'msgmodel', 'transaction_name',
               'was_missing', 'userinfo_hint']

  def __init__(self, user, transaction_name):
    if not isinstance(user, users.User):
      raise TypeError, type(user)
    if not isinstance(transaction_name, str):
      raise TypeError, type(transaction_name)
    assert user.user_id()
    self.user = user
    self.transaction_name = transaction_name
    self.user_id = str(user.user_id())
    self.mc_client = memcache.Client()
    self.msgmodel = None
    self.was_missing = False  # TODO: Make this automatic in the framework.
    self.userinfo_hint = None

  #@override db_cache.EntryEnv
  def get_cache(self):
    msgsdata = self.mc_client.gets(self.user_id, namespace='Message')
    if msgsdata is None:
      logging.info('cache miss in %s' % self.transaction_name)
      self.was_missing = True
    else:
      logging.info('cache hit in %s' % self.transaction_name)
      self.was_missing = False
    return msgsdata

  #@override db_cache.EntryEnv
  def delete_cache(self):
    logging.info('cache delete in %s' % self.transaction_name)
    return self.mc_client.delete(self.user_id, namespace='Message')

  #@override db_cache.EntryEnv
  def cas_cache(self, value):
    if not isinstance(value, str):
      raise TypeError, type(value)
    if self.was_missing:
      # self.mc_client.cas always fails on a previously missing key.
      ok = self.mc_client.add(self.user_id, value, namespace='Message')
      # Deliberately not setting: self.was_missing = False
    else:
      ok = self.mc_client.cas(self.user_id, value, namespace='Message')
    if ok:
      logging.info('cache updated in %s' % self.transaction_name)
    else:
      logging.info('cache update-too-late in %s' % self.transaction_name)
    return ok

  #@override db_cache.EntryEnv
  def get_db(self):
    msgmodels = list(Message.all().ancestor(get_user_key(self.user)))
    if msgmodels:
      assert len(msgmodels) == 1
      self.msgmodel = msgmodels[0]
      assert msgmodels[0].msgsdata, repr(msgmodels[0].msgsdata)
      return msgmodels[0].msgsdata
    else:
      self.msgmodel = None
      # It's important that we don't return None here, so sync_combiner can
      # take this '' and return '' (because it would be an error for it to
      # return None).
      return ''

  #@override db_cache.EntryEnv
  def put_db(self, value):
    msgsdata = value
    if not isinstance(msgsdata, str):
      raise TypeError, type(msgsdata)
    if not self.msgmodel:
      self.msgmodel = Message(parent=get_user_key(self.user))
    self.msgmodel.msgsdata = msgsdata
    if not msgsdata:
      self.msgmodel.delete()
    else:
      self.msgmodel.put()
    if self.userinfo_hint and self.userinfo_hint[0] == msgsdata:
      userinfo = self.userinfo_hint[1]
    else:
      logging.warn('slow userinfo computation in MessageEntryEnv')
      messages = msg_data.parse_messages(value)
      userinfo = compute_userinfo(messages, value)
      del messages
    update_user_data_entity(self.user, userinfo)

  def set_userinfo_hint(self, msgsdata, userinfo):
    self.userinfo_hint = (msgsdata, userinfo)

  #@override db_cache.EntryEnv
  def run_in_transaction(self, function, *args, **kwargs):
    call = [(run_in_counted_transaction,
             (self.transaction_name, function) + tuple(args), kwargs)]
    del function, args, kwargs
    return apply(*call.pop())


def add_message(user, new_message, transaction_name):
  """Adds a new message. To be run outside a transaction."""
  assert user.user_id()

  def do_add_message():
    parent_key = get_user_key(user)
    msgmodels = list(Message.all().ancestor(parent_key))
    if msgmodels:
      assert len(msgmodels) == 1
      msgmodel = msgmodels.pop()
      messages = list(msg_data.parse_messages(msgmodel.msgsdata))
    else:
      msgmodel = Message(parent=parent_key)
      messages = []
    messages.append(new_message)
    msgmodel.msgsdata = msg_data.serialize_messages(messages)
    userinfo = compute_userinfo(messages, msgmodel.msgsdata)
    msgmodel.put()
    update_user_data_entity(user, userinfo)
    return msgmodel.msgsdata

  msgsdata = run_in_counted_transaction(transaction_name, do_add_message)
  memcache.set(str(user.user_id()), msgsdata, namespace='Message')
  logging.info('cache extended in %s' % transaction_name)


def get_url_without_autologin(url):
  i = url.find('?')
  if i < 0:
    i = len(url)
  return url[:i] + get_updated_qstr(url[i:], 'autologin', '0', None)


def add_link(links, curpath, path, caption):
  if curpath == path:
    links.append('<b>%s</b>\n' % cgi.escape(caption))
  else:
    links.append('<a href="%s">%s</a>\n' %
                 (cgi.escape(path), cgi.escape(caption)))


def to_https(url, base=None):
  if not isinstance(url, str):
    raise TypeError
  if is_development:
    return url
  if url.startswith('/') and base and '://' in base:
    match = re.match(r'(?i)https?://([^/?#]+)', base)
    if match:
      url = 'https://%s%s' % (match.group(1), url)
  if not url.startswith('http://'):
    return url
  return 'https://' + url[7:]  # 7 == len('http://').


def get_user_with_id(request):
  if is_development:
    test_user = (request.headers.get('x-test-user') or  # Case insensitive.
                 'test@example.com')
  else:
    test_user = None
  user = users.get_current_user()
  if not user and test_user:
    user = users.User(test_user)
    if user.user_id() is None:  # For the development server.
      user = email_to_user(user.email())
    assert user.user_id() is not None
  return user


def dump_json(obj):
  return json.dumps(
    obj, ensure_ascii=False, encoding='UTF-8',
    separators=(',', ':')).encode('UTF-8')


# --- Page classes.


class Page(webapp.RequestHandler):
  def get_header_and_trailer(self, onload=None, do_add_left_login=False):
    params = {'onload': '', 'for_email': ''}
    if onload:
      params['onload'] = ' onload="%s"' % cgi.escape(onload)
    user = users.get_current_user()
    if user:
      params['for_user' ] = ' for <b>%s</b>' % user.email()
      params['loginout_link'] = '<a href="/logout">Logout</a>\n'
      links = []
      curpath = self.request.path
      add_link(links, curpath, '/list', 'Messages')
      add_link(links, curpath, '/add', 'Add message')
      add_link(links, curpath, '/count', 'Count')
      add_link(links, curpath, '/', 'Home')
      params['links'] = ''.join(links)
      del links
    else:
      params['for_user' ] = ''
      params['loginout_link'] = '<a href="/login">Login</a>\n'
      if do_add_left_login:
        if self.request.path != '/':
          params['links'] = ('<a href="/login">Login</a>\n'
                             '<a href="/">Home</a>\n')
        else:
          params['links'] = '<a href="/login">Login</a>\n'
      else:
        params['links'] = ''
    return ('<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"\n'
            '    "http://www.w3.org/TR/html4/loose.dtd">\n'
            '<html><head>\n<title>SyncStar</title>\n'
            '<meta http-equiv="Content-Type" CONTENT="text/html; '
            'charset=utf-8">\n'
            '<link rel="stylesheet" type="text/css" href="/sms.css">\n'
            '</head>\n'
            '<body%(onload)s>\n<div class=hd>'
            '<div class=l>\n'
            '%(links)s</div>\n'
            '<div class=r>\n'
            'SyncStar%(for_user)s\n'
            '%(loginout_link)s</div>\n'
            '<div style="clear:both"></div></div>' % params,
            '</body></html>\n')

  def denied(self, reason, do_add_left_login=True):
    """reason is HTML."""
    logging.error('denied: ' + str(reason))
    header, trailer = self.get_header_and_trailer(
        do_add_left_login=do_add_left_login)
    self.response.out.write('%s\n'
                            '<h1>Denied</h1>\n'
                            '%s\n%s'
                            % (header, reason, trailer))
    self.response.set_status(403)

  def bad_request(self, reason):
    """reason is HTML."""
    logging.error('bad_request: ' + str(reason))
    header, trailer = self.get_header_and_trailer()
    self.response.out.write('%s\n'
                            '<h1>Bad request</h1>\n'
                            '%s\n%s'
                            % (header, reason, trailer))
    self.response.set_status(400)

  def not_found(self, reason):
    """reason is HTML."""
    logging.error('bad_request: ' + str(reason))
    header, trailer = self.get_header_and_trailer()
    self.response.out.write('%s\n'
                            '<h1>Not found</h1>\n'
                            '%s\n%s'
                            % (header, reason, trailer))
    self.response.set_status(404)


class NotFoundPage(Page):
  def get(self):
    # self.request.query_string is not included in self.request.path
    self.not_found('The path <b>%s</b> was not found. '
                   'Try the <a href="/">Home</a> page instead.' %
                   cgi.escape(self.request.path))

  def post(self):
    self.get()


class LoginPage(Page):
  def get(self):
    self.redirect(users.create_login_url(to_https(
        self.request.referer or '/', base=self.request.uri)))


class LogoutPage(Page):
  def get(self):
    if is_development:
      self.redirect(users.create_logout_url(self.request.referer or '/')) 
      return

    # On the production instance, we just remove the session cookie, because
    # redirecting users.create_logout_url(...) would log out of all Google
    # (e.g. Gmail, Google Calendar).
    #
    # It seems that AppEngine is setting the ACSID cookie for http:// ,
    # and the SACSID cookie for https:// . We just unset both below.
    #
    # TODO: Make sure we log out from both http:// and https:// .
    cookie = Cookie.SimpleCookie()
    cookie['ACSID'] = ''
    cookie['ACSID']['expires'] = -86400  # In the past, a day ago.
    self.response.headers.add_header(*cookie.output().split(': ', 1))
    cookie = Cookie.SimpleCookie()
    cookie['SACSID'] = ''
    cookie['SACSID']['expires'] = -86400  # In the past, a day ago.
    self.response.headers.add_header(*cookie.output().split(': ', 1))
    self.redirect(self.request.referer or '/') 


class MainPage(Page):
  def get(self): 
    user = users.get_current_user()
    params = {}
    params['header'], params['trailer'] = self.get_header_and_trailer()
    if user:
      params['messages_link'] = (
          '<p><a href="/list">View your synchronized messages</a>')
    else:
      params['messages_link'] = (
          '<p><a href="/list?autologin=1">'
          'Log in and view your synchronized messages</a>')
    self.response.out.write("""%(header)s
        <p>SyncStar lets you synchronize text messages (SMS) on your Android
        mobile devices, and view those messages on the web.
        <p>For synchronization you need a mobile device (phone, tablet
        etc.), running Android 2.2 or newer, with a Google account
        set up, and the <a href="/syncstar.apk">SyncStar Android app</a>
        installed and configured.
        %(messages_link)s
        %(trailer)s""" % params)


class CountPage(Page):
  def get(self): 
    user = users.get_current_user()
    if not user:
      return self.denied('You have to be logged in to view the message count.')
    is_admin = users.is_current_user_admin()
    def do_count():
      # Counting like this is too slow (and expensive).
      counts = {'totallivemsgcount': 0, 'userlivemsgcount': 0,
                'totaldeletedmsgcount': 0, 'userdeletedmsgcount': 0,
                'totalcompressedsize': 0, 'usercompressedsize': 0,
                'totalusercount': 0, 'userusercount': 0,
                'userdeletedusercount': 0, 'totaldeletedusercount': 0,
                'usertimecreated' : 'not present',
                'usertimedeleted':'not wiped'}
      if is_admin:
        query = UserData.all()  # Expensive query, returns all entities.
      else:
        # This is a cheaper query, returns only a single entity.
        # TODO: Cache this to memcache.
        query = UserData.all().filter('user =', user)   
      for user_data in query:
        counts['totalusercount'] += 1
        if user_data.deleted:
          counts['totaldeletedusercount'] += 1
        counts['totallivemsgcount'] += user_data.livemsgcount
        counts['totaldeletedmsgcount'] += user_data.deletedmsgcount
        counts['totalcompressedsize'] += user_data.compressedsize
        if user_data.user.user_id() == user.user_id():
          counts['userusercount'] += 1
          counts['userlivemsgcount'] += user_data.livemsgcount
          counts['userdeletedmsgcount'] += user_data.deletedmsgcount
          counts['usercompressedsize'] += user_data.compressedsize
          counts['usertimecreated'] = format_mts(user_data.time_created)
          if user_data.deleted:
            counts['userdeletedusercount'] += 1
            counts['usertimedeleted'] = format_mts(user_data.time_deleted)
      return counts
    # No transaction to make it faster.
    counts = run_counted('count', do_count)
    counts['useremail'] = cgi.escape(user.email())
    counts['userispresent'] = NO_YES[counts['userusercount']]
    counts['userisdeleted'] = NO_YES[counts['userdeletedusercount']]
    if is_admin:
      total_msg = (
          'Total user count: %(totalusercount)s\n'
          '<br>Total deleted user count: %(totaldeletedusercount)s\n'
          '<br>Total live message count: %(totallivemsgcount)s\n'
          '<br>Total deleted message count: %(totaldeletedmsgcount)s\n'
          '<br>Total compressed message size: %(totalcompressedsize)s bytes\n'
          '<br>'
          % counts)
    else:
      total_msg = ''
    user_msg = (
        'You are present: %(userispresent)s\n'
        '<br>You are present since: %(usertimecreated)s\n'
        '<br>Your live message count: %(totallivemsgcount)s\n'
        '<br>Your deleted message count: %(totaldeletedmsgcount)s\n'
        '<br>Your compressed message size: %(usercompressedsize)s bytes\n'
        '<br>Messages are wiped: %(userisdeleted)s\n'
        '<br>Messages are wiped since: %(usertimedeleted)s\n'
        '<br>Your e-mail: %(useremail)s\n'
        % counts)
    header, trailer = self.get_header_and_trailer()
    self.response.out.write(
        '%s\n'
        '<p>Counts and statistics:</p><p>\n%s%s'
        '</p>\n'
        '%s'
        % (header, total_msg, user_msg, trailer))


class AddRandomPage(Page):
  def get(self):
    user = users.get_current_user()
    if user:
      addr = '+%s' % random.randint(1000000, 3000000 - 1)
      new_message = dict(
          address=addr,
          date=random.randint(1000000000000, 2000000000000 - 1),
          body=' '.join([random.choice(WORDS) for j in xrange(8)]),
          service_center='',
          error_code=0,
          locked=0,
          read=0,
          seen=0,
          protocol=0,
          status=-1,
          type=1,
          deleted=False,
          phone='webapp',
          conversation='')  # Meaning is same as addr, but don't display.
      add_message(user, new_message, 'add_random')
      self.redirect('/list')
    else:
      self.denied('You have to be logged in to add a message.')


class AddMessagePage(Page):
  def get(self):
    user = users.get_current_user()
    if user:
      # `(new Date()).getTime()' would calculate the timestamp on the client.
      header, trailer = self.get_header_and_trailer(
          onload='document.forms.sms.date.value=%d' %
          int(time.time() * 1000 + .5))
      self.response.out.write("""%s
        <p><form name="sms" action="/add" method="post">
          Address: <input type="text" name="address">
          <br>Date: <input type="text" name="date">
          <br>Body: <input type="text" name="body" size="60">
          <br>Phone: <input type="text" name="phone" value="webapp">
          <br>Conversation: <input type="text" name="conversation">
          <br>Service center: <input type="text" name="service_center">
          <br>Error code: <input type="text" name="error_code" "value="0">
          <br>Locked: <input type="text" name="locked" "value="0">
          <br>Read: <input type="text" name="read" "value="0">
          <br>Seen: <input type="text" name="seen" "value="0">        
          <br>Protocol: <input type="text" name="protocol" value="0">
          <br>Status: <input type="text" name="status" value="-1">
          <br>Type: <input type="text" name="type" value="1"> (received=1, sent=2, draft=3, outbox=4, failed=5, queued=6)
          <br><input type="submit" value="Add message">
        </form>
        %s""" % (header, trailer))
    else:
      self.denied('You have to be logged in to add a message.')
  def post(self):
    user = users.get_current_user()
    if user:
      # self.request.get returns Unicode strings, with UTF-8 being the default
      # request charset. Good.
      new_message = {}
      for name in self.request.arguments():
        value = self.request.get(name)
        if value:  # Ignore empty strings.
          new_message[str(name)] = value
      if not self.request.get('date', ''):
        new_message['date'] = int(time.time() * 1000 + .5)
      add_message(user, new_message, 'add_message')
      self.redirect('/list')
    else:
      self.denied('You have to be logged in to add a message.')


class DeleteAllPage(Page):
  def get(self):
    user = get_user_with_id(self.request)
    if not user:
      return self.denied('You have to be logged in to delete messages.')
    qstr = self.request.query_string
    qs = cgi.parse_qs(qstr, keep_blank_values=False)
    do_clear_cache = (is_development and
                      (''.join(qs.get('clearcache') or ()) or '0') != '0')
    def delete_all(messages):
      to_be_deleted = []
      for m in messages:
        if m.kind() == 'Message':
          to_be_deleted.append(m)
        elif m.kind() == 'UserData':
          m.time_deleted = int(time.time() * 1000 + .5)
          m.deleted = True
          m.livemsgcount = 0
          m.deletedmsgcount = 0
          m.compressedsize = 0
          m.put()
      db.delete(to_be_deleted)
    messages = db.Query().ancestor(get_user_key(user))
    try:
      run_in_counted_transaction('delete_all', delete_all, messages)
    finally:
      if do_clear_cache:
        memcache.delete(str(user.user_id()), namespace='Message')
      else:
        memcache.set(str(user.user_id()), '', namespace='Message')
    self.redirect('/list')


class DeletePage(Page):
  def get(self):
    user = users.get_current_user()
    if not user:
      return self.denied('You have to be logged in to delete messages.')

    message_entry_env = None

    def delete_combiner(msgsdata, _):
      userinfo = None
      if not msgsdata:
        messages = ()
      else:
        messages = msg_data.parse_messages(msgsdata)
      if messages:
        has_changed = False
        for i in message_idxs_to_delete:
          if i < len(messages) and not messages[i]['deleted']:
            messages[i]['deleted'] = True
            has_changed = True
        if has_changed:
          msgsdata = msg_data.serialize_messages(messages)
          message_entry_env.set_userinfo_hint(
              msgsdata, compute_userinfo(messages, msgsdata))
      return msgsdata, None

    message_idxs_to_delete = [i for i in map(int, self.request.get_all('d'))
                              if i >= 0]
    if message_idxs_to_delete:
      message_entry_env = MessageEntryEnv(user, 'delete')
      userinfo = db_cache.update_cache_and_db(
          combiner=delete_combiner,
          is_combiner_idempotent=True,
          delta=None,
          entry_env=message_entry_env)
      message_entry_env = None
    self.redirect('/list')
         

class JsonSpeedPage(Page):
  def get(self):
    user = users.get_current_user()
    qstr = self.request.query_string
    qs = cgi.parse_qs(qstr, keep_blank_values=False)
    is_admin = users.is_current_user_admin()
    if not (user and is_admin):
      return self.denied('You have to be logged in as an admin for this.')
      return
    message_entry_env = MessageEntryEnv(user, 'jsonspeed')
    msgsdata = message_entry_env.get_cache()
    if msgsdata is None:
      msgsdata = message_entry_env.get_db()
      message_entry_env.cas_cache(msgsdata or '')
    self.response.headers['Content-Type'] = 'text/plain; charset=UTF-8'
    if not msgsdata:
      self.response.out.write('No data.')
      return
    uncompressed = zlib.decompress(msgsdata)
    recompressed = zlib.compress(uncompressed, 9)
    messages = msg_data.parse_messages(msgsdata)
    json_str = dump_json(messages)
    json_compressed = zlib.compress(json_str, 9)
    msgpack_compressed = msg_data.serialize_messages_msgpack(messages)
    msgpack_str = zlib.decompress(msgpack_compressed)

    repeat_count = int(''.join(qs.get('rc') or ()) or '10')
    self.response.out.write('compressed_size=%d\n' % len(msgsdata))
    self.response.out.write('uncompressed_size=%d\n' % len(uncompressed))
    self.response.out.write('recompressed9_size=%d\n' % len(recompressed))
    self.response.out.write('message_count=%d\n' % len(messages))
    self.response.out.write('uncompressed_json_size=%d\n' % len(json_str))
    self.response.out.write('compressed_json_size=%d\n' % len(json_compressed))
    self.response.out.write('uncompressed_msgpack_size=%d\n' % len(msgpack_str))
    self.response.out.write('compressed_msgpack_size=%d\n' %
                            len(msgpack_compressed))
    self.response.out.write('json_version=%s\n' % json.__version__)
    self.response.out.write('json_has_speedups=%s\n' %
                            bool(getattr(json, '_speedups', None)))
    self.response.out.write('jsonparse_repeat_count=%d\n' % repeat_count)
    t0 = time.time()
    for r in xrange(repeat_count):
      json.loads(json_str)
    t1 = time.time()
    self.response.out.write('total_time_ms=%d\n' % int((t1 - t0) * 1000 + .5))
    self.response.out.write(
        'per_msg_time_us=%d\n' %
        int((t1 - t0) * 1000000 / repeat_count / len(messages) + .5))


class MessageListPage(Page):
  VERBOSE_MSGS = ('Show all columns', 'Show less columns')
  SHOWDEL_MSGS = ('Show deleted messages', 'Hide deleted messages')

  def get(self):
    user = users.get_current_user()
    qstr = self.request.query_string
    qs = cgi.parse_qs(qstr, keep_blank_values=False)
    if not user:
      do_autologin = (''.join(qs.get('autologin') or ()) or '0') != '0'
      if do_autologin:
        url = get_url_without_autologin(self.request.uri)
        self.redirect(users.create_login_url(url))
      else:
        self.denied('You have to be logged in to list messages.')
      return
    if 'autologin' in qs:
      self.redirect(to_https(get_url_without_autologin(self.request.uri)))
    def do_list():
      output = []
      message_entry_env = MessageEntryEnv(user, 'list')
      msgsdata = message_entry_env.get_cache()
      if msgsdata is None:
        msgsdata = message_entry_env.get_db()
        message_entry_env.cas_cache(msgsdata or '')
      if not msgsdata:
        messages = ()
      else:
        messages = msg_data.parse_messages(msgsdata)
      del message_entry_env, msgsdata
      i = -1
      for message in messages:
        i += 1
        message['idx'] = i
      messages = sorted(messages,
                        key=lambda message: -message['date'])  # Latest first.
      # self.request.query_string is an str (not unicode).
      is_verbose = (''.join(qs.get('verbose') or ()) or '0') != '0'
      do_show_deleted = (''.join(qs.get('showdel') or ()) or '0') != '0'
      header, trailer = self.get_header_and_trailer()
      output.append("""%s<p>
        <form action="/addrandom" method="get">
          <input type="submit" value="Add random message">
        </form>
        <form action="/add" method="get">
          <input type="submit" value="Add custom message">
        </form>
        <form action="/deleteall" method="get">
          <input type="submit" value="Wipe all messages from the web"> (Phones are unaffected.)
        </form>
        """ % header)
      parent_user = UserData.get_by_key_name(user.user_id())
      if parent_user:
        header_params = {
            'qs_verbose': cgi.escape(get_updated_qstr(qstr, 'verbose', '0', not is_verbose)),
            'qs_showdel': cgi.escape(get_updated_qstr(qstr, 'showdel', '0', not do_show_deleted)),
            'msg_verbose': cgi.escape(self.VERBOSE_MSGS[is_verbose]),
            'msg_showdel': cgi.escape(self.SHOWDEL_MSGS[do_show_deleted]),
        }
        output.append("""
          <p><a href="/list%(qs_verbose)s">%(msg_verbose)s</a>
          <a href="/list%(qs_showdel)s">%(msg_showdel)s</a>
          <form action="/delete" method="get"></p>
          <table class="m">
             <tr>
               <th>Delete</th>
               <th>Date</th>
               <th>Type</th>
               <th>Address</th>
               <th>Body</th>
        """ % header_params)
      if is_verbose:
        output.append("""
            <th>Conversation</th>        
            <th>Phone</th>
            <th>Service center</th>
            <th>Error code</th>
            <th>Locked</th>
            <th>Read</th>
            <th>Seen</th>
            <th>Protocol</th>
            <th>Status</th>
            """)
      output.append('</tr>\n')
      live_count = 0
      deleted_count = 0
      i = -1
      for message in messages:
        i += 1
        if message['deleted']:
          deleted_count += 1
          if not do_show_deleted:
            i -= 1
            continue
        else:
          live_count += 1
        params = {'idx': message['idx'],
                  'address': cgi.escape(message['address']),
                  'date': format_mts(message['date']),
                  'body': cgi.escape(message['body']),
                  'conversation': cgi.escape(message['conversation']),
                  'phone': cgi.escape(message['phone'] or ''),
                  'service_center': message['service_center'],
                  'error_code': message['error_code'],
                  'locked': message['locked'],
                  'read': message['read'],
                  'seen': message['seen'],
                  'protocol': message['protocol'],
                  'status': message['status'],
                  'type': TYPE_TO_STRING.get(message['type']) or
                          str(message['type'])}
        if message['deleted']:
          params['delete'] = 'deleted'
        else:
          # Use the short name d to avoid long query strings in the URL.
          # TODO: Generate a comma-separated list from JavaScript.
          params['delete'] = (
              '<input type="checkbox" name="d" value="%(idx)s">' % params)
        if i % 2:
          params['trclass'] = ' class=o'
        else:
          params['trclass'] = ''
        output.append("""
            <tr%(trclass)s>
            <td>%(delete)s</td>
            <td>%(date)s</td>
            <td>%(type)s</td>
            <td>%(address)s</td>
            <td>%(body)s</td>\n""" % params)
        if is_verbose:
          output.append("""
              <td>%(conversation)s</td>
              <td>%(phone)s</td>         
              <td>%(service_center)s</td>
              <td>%(error_code)s</td>
              <td>%(locked)s</td>
              <td>%(read)s</td>
              <td>%(seen)s</td>
              <td>%(protocol)s</td>
              <td>%(status)s</td>
              </tr>\n""" % params)
      # This would fetch less data, but it would also need an index on
      # (deleted).
      # deleted_count = Message.all(keys_only=True).filter('deleted = ', True).filter('user = ', user), count()
      output.append("""
        </table>
        <p><input type="submit" value="Delete selected above">
        </form>
        <p>%(live_count)s messages live, %(deleted_count)s deleted.</p>
        %(trailer)s""" % {'live_count': live_count,
                          'deleted_count': deleted_count,
                          'trailer': trailer})
      return output

    # If this gets converted to a transaction, we mustn't call
    # self.response.write within the transaction.
    output = run_counted('list', do_list)
    self.response.out.write(''.join(output))


class SyncPage(Page):
  def _short_msg(self, msg, status, default_msg):
    msg = msg_data.unicode_to_utf8(msg)
    if not msg.endswith('\n'):
      msg += '\n'
    if msg == '\n':
      msg = default_msg
      if not msg.endswith('\n'):
        msg += '\n'
    self.response.headers['Content-Type'] = 'text/plain; charset=UTF-8'
    self.response.out.write(msg)
    self.response.set_status(status)

  #@Override Page
  def denied(self, msg):
    self._short_msg(msg, 403, 'Permission denied.\n')
  
  #@Override Page
  def bad_request(self, msg):
    self._short_msg(msg, 400, 'Bad request.\n')
  
  def get(self):
    user = get_user_with_id(self.request)
    if not user:
      return self.denied('You have to be logged in to sync messages.')
    qs = cgi.parse_qs(self.request.query_string, keep_blank_values=False)
    do_dump = (''.join(qs.get('dump') or ()) or '0') != '0'
    if is_development and do_dump:
      return self.dump(user)
    self.response.out.write('To synchronize, please use your Android device '
                            'to POST your messages in JSON.\n')

  def dump(self, user):
    message_entry_env = MessageEntryEnv(user, 'sync')
    # Don't use the cache for dumping.
    #msgsdata = message_entry_env.get_cache()
    #if msgsdata is None:
    msgsdata = message_entry_env.get_db()
    # message_entry_env.cas_cache(msgsdata or '')
    if msgsdata in (None, ''):
      messages = None
    else:
      messages = msg_data.parse_messages(msgsdata)
    del message_entry_env, msgsdata
    self.response.headers['Content-Type'] = 'application/json; charset=UTF-8'
    self.response.out.write(dump_json(messages))

  def post(self):
    logging.info('sync post')
    user = get_user_with_id(self.request)
    if not user:
      # TODO: Simplify error pages for /sync.
      return self.denied('You have to be logged in to sync messages.') 
    qs = cgi.parse_qs(self.request.query_string, keep_blank_values=False)
    is_request_compressed = self.request.headers.get(
        'Content-Encoding', '').lower() == 'deflate'
    body = self.request.body
    if is_request_compressed:
      if not re.match(r'[\r\n \t{\[]', body):  # Doesn't look like uncompressed.
        try:
          body = zlib.decompress(body)
        except zlib.error:
          return self.bad_request('Cannot decompress deflate stream.')
    if not body or not NONWHITESPACE_RE.search(body):
      return self.bad_request('Received empty post data. Please send request in JSON format.')

    # Disable debugging in production.
    # Don't use 'debug', because the AppEngine dev_appserver.py adds extra
    # JavaScript to the output.
    #do_debug = bool(user and ''.join(qs.get('mydebug') or ()))
    #if do_debug:
    #  logging.error('request headers=%s body=%r decompressed_body=%r' %
    #                (self.request.headers, self.request.body, body))
    logging.info('sync parse_json')
    # The sync operation needs memory proprotional to the sum of the request
    # and the user's SMS data in the datastore.
    try:
      req = json.loads(body)
    except json.JSONDecodeError, e:
      return self.bad_request('Expected messages in JSON format.\n%s' % e)
    # Validity checks.
    if not isinstance(req, dict):
      return self.bad_request('Expected dict.')
    phone = req.get('phone')
    if not isinstance(phone, (str, unicode)):
      return self.bad_request('Expected string in phone.')
    if not phone:
      return self.bad_request('Expected non-empty string in phone.')
    phone = msg_data.utf8_to_unicode(phone)
    if not isinstance(req.get('messages'), list):
      return self.bad_request('Expected list in messages.')
    input_messages = {}
    
    logging.info('sync parse_messages')
    i = -1
    for message in req['messages']:
      i += 1
      if not isinstance(message, dict):
        return self.bad_request('Expected list of dict in messages.')
      if 'phone' in message:
        # TODO: Allow this in case restoring after a wipe.
        return self.bad_request('Unexpected phone.')
      errors = msg_data.fix_sync_input_message(message)
      if errors:
        return self.bad_request(errors[0])
      message['phone'] = phone
      message['i'] = i
      if message['conversation'] == message['address']:
        message['conversation'] = ''  # Save space.
      dedupid = (message['address'], message['body'],
                 message['date'], message['type'])
      input_messages.setdefault(dedupid, message)  # Keep the lowest index.
    # TODO: This function can be slow, measure the time for each subpart.
    del req

    message_entry_env = MessageEntryEnv(user, 'sync')

    def sync_combiner(msgsdata, input_messages):
      # msgsdata is the Value
      # input_messages is the Delta.
      assert isinstance(input_messages, dict), type(input_messages)
      if not msgsdata:
        messages = ()
      else:
        messages = msg_data.parse_messages(msgsdata)
      logging.info('sync increase')
      update_count = 0
      messages_to_return = []
      for message in messages:
        dedupid = (message['address'], message['body'], message['date'],
                   message['type'])
        fields = input_messages.get(dedupid)
        if fields:
          if (message['read'] < fields['read'] or
              message['seen'] < fields['seen'] or
              message['deleted'] < fields['deleted']):
            message['read'] = max(message['read'], fields['read'])
            message['seen'] = max(message['seen'], fields['seen'])
            message['deleted'] = max(message['deleted'], fields['deleted'])
            update_count += 1
          if (message['read'] > fields['read'] or
              message['seen'] > fields['seen'] or
              message['deleted'] > fields['deleted']):
            messages_to_return.append(message)
          del input_messages[dedupid]
        elif not message['deleted']:
          messages_to_return.append(message)
      messages_to_insert = input_messages.values()
      del input_messages
      # Without this our operation isn't deterministic, because it depends on
      # the hash iteration order.
      messages_to_insert.sort(key=lambda message: message['i'])

      logging.info('sync insert#=%d update#=%d return#=%d' %
                   (len(messages_to_insert), update_count,
                    len(messages_to_return)))
      if messages_to_insert or update_count:
        messages = list(messages)
        # Don't reorder messages, because we use use the index for deletion
        # (in class DeletePage).
        messages.extend(messages_to_insert)
        # This ingores all extra fields (such as "i") in messages.
        msgsdata = msg_data.serialize_messages(messages)
        message_entry_env.set_userinfo_hint(
            msgsdata, compute_userinfo(messages, msgsdata))
        return msgsdata, messages_to_return
      else:
        return msgsdata, messages_to_return

    # TODO: Better log cache hits etc.
    messages_to_return = db_cache.update_cache_and_db(
        combiner=sync_combiner,
        is_combiner_idempotent=True,
        delta=input_messages,
        entry_env=message_entry_env)
    assert isinstance(messages_to_return, list), type(messages_to_return)
    del input_messages
    message_entry_env = None

    # Generate the HTTP response.
    logging.info('sync build_response')
    # This is superfluous, should have been removed earlier.
    for message in messages_to_return:
      if (message['conversation'] and
          message['conversation'] == message['address']):
        message['conversation'] = ''
    
    def message_to_response_item(message):
      # TODO: Don't return all the fields (only address, body, date, type, read,
      #       seen, deleted) for updated messages. Update the documentation.
      msginfo = {}
      for name in ('address', 'body', 'date', 'type', 'service_center',
                   'error_code', 'locked', 'read', 'seen', 'protocol',
                   'phone', 'deleted'):
        if message[name]:  # 0, False or '' is the default.
          msginfo[name] = message[name]
      if message['status'] != -1:
        msginfo['status'] = message['status']
      if (message['conversation'] and
          message['conversation'] != message['address']):
        msginfo['conversation'] = message['conversation']
      return msginfo
    res = dump_json({'smessages':
                     map(message_to_response_item, messages_to_return)})
    self.response.headers['Content-Type'] = 'application/json; charset=UTF-8'
    if is_request_compressed and len(res) > 128:  # Compress the response.
      logging.info('sync compress')
      # AppEngine removes Content-Encoding from the response header, so we
      # use X-Content-Encoding here.
      # See http://code.google.com/p/googleappengine/issues/detail?id=1378 .
      self.response.headers['X-Content-Encoding'] = 'deflate'
      res = zlib.compress(res, 9)  # Highest compression ratio.
    logging.info('sync done')
    self.response.out.write(res)


application = webapp.WSGIApplication([('/', MainPage),
                                      ('/list', MessageListPage),
                                      ('/login', LoginPage),
                                      ('/logout', LogoutPage),
                                      ('/add', AddMessagePage),
                                      ('/sync', SyncPage),
                                      ('/addrandom', AddRandomPage),
                                      ('/count', CountPage),
                                      ('/delete', DeletePage),
                                      ('/deleteall', DeleteAllPage),
                                      ('/jsonspeed', JsonSpeedPage),
                                      ('.*', NotFoundPage)],
                                      debug=is_development)


def main():
  #logging.getLogger().setLevel(logging.WARN)
  logging.getLogger().setLevel(logging.INFO)
  run_wsgi_app(application)

if __name__ == '__main__':
  main()
